# -*- coding: utf-8 -*-
"""
Created on Thu Aug  4 12:00:11 2016

@author: XT-YY

get market data from wind and SoHu

get netvalue data from eastmoney
"""
import pandas as pd 
import os
import time
import urllib, json
from bs4 import BeautifulSoup
from splinter import Browser
from WindPy import w



dst_path = r".\\temp"


def dateTransform(date):
    return date[0:4]+"-"+date[4:6]+"-"+date[6:8]


def get_wind():
    if not w.isconnected():
        w.start()
    return w

if __name__ == "__main__":
    sdate = "20150101"
    edate = "20160804"
    
    dict_stock = {}
    
       
    # get market data from wind    
    code_list = ['511880.SH', '511990.SH']    
    while(code_list):
        code = code_list[0]
        d = get_wind().wsd(code, 'open, high, low, close, volume, amt', sdate, edate, 'Fill=Previous')
        if d.ErrorCode != 0:
            continue
        code_list.remove(code)
        df = pd.DataFrame()
        df["#Code"] = [code]*len(d.Times)
        df["Date"] = [x.strftime("%Y-%m-%d")  for x in d.Times]
        df["Open"] = d.Data[0]
        df["High"] = d.Data[1]
        df["Low"] = d.Data[2]
        df["Close"] = d.Data[3]
        df["Volume"] = d.Data[4]
        df["Amount"] = d.Data[5]
        dict_stock[code] = df
        time.sleep(5)
        
    # get market data from sohu
    col_name = ["Date", "Open", "Close", "Change", "PCT_Change", "Low", "High", 
                "Volume", "Amount", "TurnOver"]
    code = "204001"
    url = ("http://q.stock.sohu.com/hisHq?code=cn_" + code + "&start=" 
              + sdate + "&end=" + edate + "&stat=1&order=D&period=d")
    response = urllib.request.urlopen(url);
    time.sleep(1)
    text = response.read().decode("gbk")
    data = json.loads(text, encoding="gbk")
    data = data[0]['hq']
    df = pd.DataFrame(data)
    
    df = df.iloc[:,[0,1,2,5,6,7,8]]
    df.columns = ["Date", "Open", "Close", "Low", "High", "Volume", "Amount"] 
    code = code + ".SH"
    df["#Code"] = code
    df = df[["#Code", "Date", "Open", "High", "Low", "Close", "Volume", "Amount"]]
    df.sort_values(by="Date", ascending=True, inplace=True)
    dict_stock[code] = df
    

    # get netvalue data from eastmoney
    
    code = "511880"
    url = ("http://fund.eastmoney.com/f10/F10DataApi.aspx?type=lsjz&code=" + code
        + "&page=1&per=10000&sdate=" + dateTransform(sdate) + "&edate=" + dateTransform(edate))
    browser = Browser('phantomjs', user_agent="Mozilla/5.0 (Windows NT 6.1; WOW64) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36")
    browser.driver.maximize_window()
    browser.visit(url)
    time.sleep(3)
    xpath = '/html/body/table/tbody'
    table = browser.find_by_xpath(xpath).first.text.split("\n")
    data = [x.split(' ')[:3] for x in table]
    col_name = ["Date", "NetValue", "AccumulativeNetValue"]
    df = pd.DataFrame(data, columns=col_name)
    code = code + ".SH"
    ndf = pd.merge(dict_stock[code], df, on="Date", how="outer")
    dict_stock[code] = ndf
    
    # generate file
    for k,v in dict_stock.items():
        v.to_csv(dst_path+os.sep+k+".csv", index=False)
        
    
    
    
        
    

    